/**
 * @file PageLibPreprocessor.hpp
 * @author your name (you@domain.com)
 * @brief
 * @version 0.1
 * @date 2022-04-05
 *
 * @copyright Copyright (c) 2022
 *
 */

#pragma once

#include "WebPage.hpp"
#include "WordSegmentation.hpp"

#include <iostream>
#include <map>
#include <set>
#include <string>
#include <vector>

using std::cout;
using std::endl;
using std::map;
using std::pair;
using std::set;
using std::string;
using std::unordered_map;
using std::vector;

using namespace wd;

namespace wd
{
class Configuration;

class PageLibPreprocessor
{
  public:
    PageLibPreprocessor(Configuration &conf);

    void doProcess();

  private:
    void readInfoFromFile();      // 从配置信息读取网页库和偏移库的内容
    void cutRedundantPages();     // 去重
    void buildInvertIndexTable(); // 建立倒排索引表
    void storeOnDisk();           // 写回磁盘 newpagelib.dat newoffset.lib invertindexlib.dat

  private:
    Configuration &_conf;
    WordSegmentation _jieba; // 对象 强关联关系

    vector<WebPage> _pageLib; // 网页库容器
    unordered_map<int, pair<int, int>> _offsetLib;
    unordered_map<string, vector<pair<int, double>>> _invertIndexTable;
};

} // end of namespace wd
